﻿using System.Threading;
using DM4;
using sscore;
using System;
using System.Collections.Generic;
using sscore.Utils;
using System.Text.RegularExpressions;
using System.IO;
using sscore.Requests;

namespace FL
{
    public class FL : ModuleBase
    {
        private readonly string _searchUrl;
        private readonly string _offenderLink;
        private readonly string _imageLink;
        private readonly string _vehicleLink;
        private const string StateName = "FL";
        private ResultFile _imagesResultFile;
        readonly string[] _searchParam = { "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z" };

        public FL()
        {
            //initialize main system constants
            ModuleCode = "SO_FL";
            Description = "FL Sex Offenders";
            Version = "1.0";

            //for creating sessions
            StartParam = 0;
            StopParam = _searchParam.Length - 1;

            //initialize help 
            AddToHelp(DM4Consts.StartPosition, String.Format("{0}...{1}", StartParam, StopParam));
            AddToHelp(DM4Consts.EndPositon, String.Format("{0}...{1}", StartParam, StopParam));

            AddToHelp(DM4Consts.IsDownload, "true|false");
            AddToHelp(DM4Consts.IsDownloadImg, "true|false");
            AddToHelp(DM4Consts.InsertToDb, "false|true", "(Expected: servername, dbname)");
            AddToHelp(DM4Consts.ServerName, "");
            AddToHelp(DM4Consts.DbName, "");
            AddToHelp(DM4Consts.IsClearDb, "false|true");
            AddToHelp(DM4Consts.TestParam, "false|true");

            Rpm.Codepage = EncodingType.Iso88591;
            RootUrl = "http://offender.fdle.state.fl.us/";
            _searchUrl = string.Format("{0}offender/offenderSearchNav.do", RootUrl);
            _offenderLink = string.Format("{0}offender/flyer.do?personId={{0}}", RootUrl);
            _imageLink = string.Format("{0}offender/CallImage?imgID={{0}}", RootUrl);
            _vehicleLink = string.Format("{0}offender/flyerVehicleVesselInfo.do?personId={{0}}", RootUrl);
        }

        private void SetParametersDefaults()
        {
            Params.CreateBooleanParam(DM4Consts.IsDownload, true);
            Params.CreateBooleanParam(DM4Consts.IsDownloadImg, true);
            Params.CreateBooleanParam(DM4Consts.InsertToDb, false);
            Params.CreateBooleanParam(DM4Consts.IsClearDb, false);
            Params.CreateStringParam(DM4Consts.DbName, "");
            Params.CreateStringParam(DM4Consts.ServerName, "");
            Params.CreateIntegerParam(DM4Consts.StartPosition, StartParam);
            Params.CreateIntegerParam(DM4Consts.LastPosition, Params.GetIntegerParam(DmConsts.StartPosition));
            Params.CreateIntegerParam(DM4Consts.EndPositon, StopParam);
            Params.CreateIntegerParam(DM4Consts.LastRecord, 0);
        }

        public override OperationResult Init(IDM4Core iCore, DmTaskParams param)
        {
            //use basic initialization sequence
            base.Init(iCore, param);

            SetParametersDefaults();
            return new OperationResult { Result = DM4Consts.Ok };
        }

        public override OperationResult Start()
        {
            DoWork();
            return new OperationResult { Result = DM4Consts.Ok };
        }

        public override OperationResult DoWork()
        {
            OperationResult or = new OperationResult { Result = DM4Consts.ErrorUnknown };

            if (!Ct.IsCancellationRequested)
            {
                WriteLog("Start parameters:", MessageType.Warning);
                foreach (KeyValuePair<string, string> kvp in Params.GetDictionary()) //get all existing parameters
                    WriteLog(String.Format("Key = {0}, Value = {1}", kvp.Key, kvp.Value), MessageType.Warning);

                if (Params.GetBooleanParam(DM4Consts.IsDownload))
                    Load();

                if (Params.GetBooleanParam(DM4Consts.InsertToDb))
                    if ((Params.GetStringParam(DM4Consts.DbName) == "") || (Params.GetStringParam(DM4Consts.ServerName) == ""))
                    {
                        or.Message = String.Format("{0} or {1} is empty", DM4Consts.DbName, DM4Consts.ServerName);
                        ResultCode = DM4Consts.SessionResultCodeError;
                        ResultMessage = or.Message;
                        UpdateStatus(DmConsts.Error);
                        WriteErrorLog(or.Message); //writting to error and session log   

                        return or;
                    }
                    else
                    {
                        //insert offenders to db 
                        UpdateStatus(DM4Consts.SessionInsertToDb);

                        Offenders.OffInsertToDatabase(StateName, Params.GetStringParam(DmConsts.DbName),
                             Params.GetStringParam(DmConsts.ServerName), ResultDir, Params.GetBooleanParam(DmConsts.IsClearDb),
                             WriteLog);

                    }

                //check if need abort task
                if (Ct.IsCancellationRequested)
                {
                    ResultCode = DM4Consts.SessionResultTerminateByUser;
                    ResultMessage = "Session terminated by user";
                    UpdateStatus(DM4Consts.SessionWait);
                    WriteLog("Session aborted by user", MessageType.Cool);
                }
                else
                {
                    UpdateProgress(1);
                    ResultCode = DM4Consts.SessionResultCodeOk;
                    ResultMessage = String.Format("Session complete, errors : {0}, records : {1}", TaskParams.ErrorCount, TaskParams.RecordCount);
                    UpdateStatus(DM4Consts.SessionComplete);
                    WriteLog("------------Finish----------------", MessageType.Cool);
                }
            }
            else
            {
                ResultCode = DM4Consts.SessionResultTerminateByUser;
                ResultMessage = "Session terminated by user";
                UpdateStatus(DM4Consts.SessionWait);
                WriteLog("Session aborted by user", MessageType.Cool);
            }


            return or;
        }

        private void Load()
        {
            string resultFileName = String.Format("{0}\\{1}.{2}", ResultDir, TaskParams.SessionId, DmConsts.ResultFilesExtension);
            bool workFlag = true;

            ResultFile = new ResultFile(resultFileName, OffendersResult.GetHeader());
            _imagesResultFile = new ResultFile(String.Format("{0}\\{1}_images.{2}", ResultDir, TaskParams.SessionId, DM4Consts.ResultFilesExtension), "FileName|FileUrl");
            try
            {
                while (workFlag)
                {
                    //рабочий цикл
                    string error;

                    try
                    {
                        error = "";  //Required!!  /*if (!String.IsNullOrEmpty(error)) --> restart thread; else --> download finished*/

                        WriteLog("Start download...");
                        Searching();
                    }
                    catch (Exception e)
                    {
                        error = e.Message;
                    }

                    if (!String.IsNullOrEmpty(error))
                    {
                        Thread.Sleep(30000);
                        WriteErrorLog(error);
                        WriteLog("Try continue work...", MessageType.Warning);
                    }
                    else
                    {
                        workFlag = false;  //все нормально откачало, заканчиваем работу
                    }
                }
            }
            finally
            {
                ResultFile.Close();
                _imagesResultFile.Close();
            }
        }

        private void GetCookies()
        {
            string html = GetRequest.ReturnCookies(_searchUrl + "?link=advanced", Rpm);
            if (!RegExpr.IsMatch(html, @"(?si:Florida Department of Law Enforcement)"))
                throw new Exception(DmConsts.ExcNeedRestart);
        }

        private void Searching()
        {
            if (Test)
            {
                Params.SetIntegerParam(DmConsts.LastPosition, Params.GetIntegerParam(DmConsts.StartPosition));
                Params.SetIntegerParam(DmConsts.EndPositon, Params.GetIntegerParam(DmConsts.StartPosition));
            }

            if (Params.GetIntegerParam(DmConsts.EndPositon) > StopParam)
                Params.SetIntegerParam(DmConsts.EndPositon, StopParam);


            for (int indSearchParam = Params.GetIntegerParam(DmConsts.LastPosition); indSearchParam <= Params.GetIntegerParam(DmConsts.EndPositon); indSearchParam++)
            {
                //check if need abort task
                if (Ct.IsCancellationRequested)
                    break;

                Params.SetIntegerParam(DmConsts.LastPosition, indSearchParam);
                UpdateProgress();

                WriteLog(string.Format("Process search param [{0}][{1}].",
                    Params.GetIntegerParam(DmConsts.LastPosition),
                    _searchParam[Params.GetIntegerParam(DmConsts.LastPosition)]));

                GetCookies();
                string html = PostRequest.RefererWithCookies(
                    String.Format("{0}; {1}", _searchUrl, RegExpr.Parse(Rpm.Cookies, "(?si:(JSESSIONID=.*?); path=)")),
                    _searchUrl + "?link=advanced",
                    String.Format("firstName=&lastName={0}&city=&county=&outOfFloridaCounty=&zip=&offenderType=3&lglStatus_1=1&lglStatus_2=2&lglStatus_3=3&lglStatus_4=4&lglStatus_5=5&lglStatus_6=6&lglStatus_7=7&lglStatus_8=8&lglStatus_9=9&stateStatus=3&link=doSearch&commaSeparatedOffenderStatus=1%2C2%2C3%2C4%2C5%2C6%2C7%2C8%2C9&commaSeparatedPersonIds=",
                        _searchParam[Params.GetIntegerParam(DmConsts.LastPosition)]),
                    Rpm);

                if (!RegExpr.IsMatch(html, @"(?si:Florida Department of Law Enforcement)"))
                    throw new Exception(DmConsts.ExcNeedRestart);

                ParseResultPage(html);
            }
        }

        private void ParseResultPage(string html)
        {
            html = HtmlUtils.KillSpace(HtmlUtils.ReplaceSpecSymbols(html.Replace("\r\n", " ").Replace("\t", " ").Replace("\r", " ").Replace("\n", " ")));

            if (RegExpr.IsMatch(html, @"(?si:Invalid PersonId)"))
                throw new Exception("Invalid PersonId on ResultPage");

            string[] arrIDs = RegExpr.Parse(html, @"(?si:name=""commaSeparatedPersonIdsALL"" value=""([^""]+)"")").
                 Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries);
            WriteLog(String.Format("Total Sex Offenders found:  [{0}]", arrIDs.Length));

            List<string> list = new List<string>();

            foreach (string id in arrIDs)
                if (!list.Contains(id))
                    list.Add(id);

            WriteLog(String.Format("found [{0}] id[s]", list.Count));

            for (int i = Params.GetIntegerParam(DmConsts.LastRecord); i < list.Count; i++)
            {
                //check if need abort task
                if (Ct.IsCancellationRequested)
                    break;

                Params.SetIntegerParam(DmConsts.LastRecord, i);
                string lookupKey = list[i];
                string offenderHtml = GetRequest.WithCookies(string.Format(_offenderLink, lookupKey), Rpm);
                WriteLog(String.Format("ParseOffender [{0}][{1}]", Params.GetIntegerParam(DmConsts.LastRecord), lookupKey));

                if (!RegExpr.IsMatch(offenderHtml, @"(?si:If further information is needed, please contact the Florida Department of Law Enforcement Registration)"))
                    throw new Exception("Offenders page:" + DmConsts.ExcNeedRestart);

                if (RegExpr.IsMatch(offenderHtml, @"(?si:Invalid PersonId)"))
                    throw new Exception("Invalid PersonId on Offenders page");
                
                ParseOffender(offenderHtml, lookupKey);
                UpdateRecordCount(1);
            }
            Params.SetIntegerParam(DmConsts.LastRecord, 0);
        }

        private void ParseOffender(string offenderHtml, string lookupKey)
        {            
            offenderHtml = HtmlUtils.KillSpace(HtmlUtils.ReplaceSpecSymbols(offenderHtml.Replace("\r\n", " ").Replace("\t", " ").Replace("\r", " ").Replace("\n", " ")));
            using (OffendersResult or = new OffendersResult())
            {
                or.Source_State = StateName;
                or.offenderLink = string.Format(_offenderLink, lookupKey);
                or.Source_id = lookupKey;

                or.fullname = RegExpr.Parse(offenderHtml, @"(?si:Name\s*:\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.AKA = RegExpr.Parse(offenderHtml, @"(?si:Aliases\s*</span>\s*</strong>\s*</div>\s*</td>\s*</tr>\s*<tr>\s*<td[^<>]*?>(.*?)</td>)").Trim();
                if (or.AKA == "Not Available") 
                    or.AKA = "";

                or.Address = RegExpr.Parse(offenderHtml, @"(?si:Map Link\s*</td>\s*</tr>\s*<tr>\s*<td[^<>]*?>(.*?)</td>)").Trim();
                or.Address = RegExpr.Replace(or.Address, @"(?si:<[/]*?\s*BR[^<>]*?>)", "@").Trim();
                or.Address_type = RegExpr.Parse(offenderHtml, @"(?si:Type of Address\s*:\s*</strong>(.*?)</td>)").Trim();
                or.Address_date = RegExpr.Parse(offenderHtml, @"(?si:Received\s*:\s*</strong>([^<>]*?)<)").Trim();
                or.Birthdate = RegExpr.Parse(offenderHtml, @"(?si:Date of Birth\s*:\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.Ethnicity = RegExpr.Parse(offenderHtml, @"(?si:Race\s*:\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.eye_color = RegExpr.Parse(offenderHtml, @"(?si:Eyes\s*:\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.hair_color = RegExpr.Parse(offenderHtml, @"(?si:Hair\s*:\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.height = RegExpr.Parse(offenderHtml, @"(?si:Height\s*:\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.weight = RegExpr.Parse(offenderHtml, @"(?si:Weight\s*:\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.sex = RegExpr.Parse(offenderHtml, @"(?si:Sex\s*:\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.marks = RegExpr.Parse(offenderHtml, @"(?si:Tattoos\s*</span>\s*</strong>\s*</div>\s*</td>\s*</tr>\s*<tr>\s*<td[^<>]*?>(.*?)</td>)").Trim();
                if (or.marks == "Information temporarily unavailable") 
                    or.marks = "";               

                or.Status = RegExpr.Parse(offenderHtml, @"(?si:Status\s*:\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.Status = RegExpr.Replace(or.Status, @"(?si:<[/]*?a[^<>]*?>)", "").Trim();

                or.Crime += "Adjudication Date; Crime Description; Court Case Number; Jurisdiction and State; Adjudication~";
                foreach (Match matchLoc in RegExpr.ParseMatchCollection(
                        RegExpr.Parse(offenderHtml, @"(?si:Adjudication\s*</td>\s*</tr>(.*?)</table>)"),
                        @"(?si:<tr[^<>]*?>(.*?)</tr>)"))
                    for (int i = 0; i < 5; i++)
                    {
                        string endStr = i == 4 ? "~" : ";";
                        try
                        {
                            if (i == 1)
                                or.Crime += RegExpr.Parse(
                                    RegExpr.ParseMatchCollection(matchLoc.Groups[1].Value, @"(?si:<td>(.*?)</td>)")[i].Groups[1].Value,
                                    @"(?si:\x29"">(.*?)</a>)").Trim() + endStr;
                            else
                                or.Crime += RegExpr.ParseMatchCollection(matchLoc.Groups[1].Value, @"(?si:<td>(.*?)</td>)")[i].Groups[1].Value.Trim() + endStr;
                        }
                        catch { or.Crime += endStr; }
                    }

                or.Crime = RegExpr.Replace(or.Crime, @"<[/]*?a[^<>]*?>", "");
            
                or.Risk = RegExpr.Parse(offenderHtml, @"(?si:Designation\s*:\s*</td>\s*<td>(.*?)</td>)").Trim();
                or.Risk = RegExpr.Replace(or.Risk, @"(?si:<[/]*?a[^<>]*?>)", "").Trim();

                or.address_source = RegExpr.Parse(offenderHtml, @"(?si:Source\s*:\s*</strong>(.*?)<br>)").Trim();
                or.victim_minor = RegExpr.Parse(offenderHtml, @"(?si:Gender\s*:\s*</strong>(.*?)<strong>)").Trim();
                or.victim_sex = RegExpr.Parse(offenderHtml, @"(?si:Minor\s*:\s*</strong>(.*?)<br>)").Trim();

                string htmlImgUrl = RegExpr.Parse(offenderHtml, @"(?si:offender/CallImage\x3FimgID=(.*?)"")");
                if (Params.GetBooleanParam(DmConsts.IsDownloadImg) && htmlImgUrl != "")
                {
                    or.Image_ID = Guid.NewGuid() + ".jpg";
                    or.ImageLink = string.Format(_imageLink, htmlImgUrl);
                    Offenders.DownloadImageWithExceptions(_imagesResultFile, String.Format("{0}\\{1}\\{2}", ResultDir, DmConsts.PhotosDir, or.Image_ID), 
                        or.ImageLink, TaskParams.SessionId);
                }

                if (or.Image_ID != "" && !File.Exists(String.Format("{0}\\{1}\\{2}", ResultDir, DmConsts.PhotosDir, or.Image_ID)))
                    throw new Exception(DmConsts.ExcImgNotFound);

                WriteLog("Navigate to vehicle and vessel page");
                offenderHtml = GetRequest.WithCookies(string.Format(_vehicleLink, lookupKey), Rpm);
                offenderHtml = HtmlUtils.KillSpace(HtmlUtils.ReplaceSpecSymbols(offenderHtml.Replace("\r\n", " ").Replace("\t", " ").Replace("\r", " ").Replace("\n", " ")));
                foreach (Match m in RegExpr.ParseMatchCollection(offenderHtml,
                    @"(?si:<tr[^<>]*?>\s*<td[^<>]*?>\s*Vehicle\s*Make\s*:\s*<[/]*?\s*BR[^<>]*?>\s*<strong>([^<>]*?)</strong>\s*</td>\s*<td[^<>]*?>\s*Vehicle\s*Type\s*:\s*<[/]*?\s*BR[^<>]*?>\s*<strong>([^<>]*?)</strong>\s*</td>\s*<td[^<>]*?>\s*Vehicle\s*Color\s*:\s*\s*<[/]*?\s*BR[^<>]*?>\s*<strong>([^<>]*?)</strong>\s*</td>\s*</tr>\s*<tr[^<>]*?>\s*<td>\s*Vehicle\s*Year\s*:\s*<[/]*?\s*BR[^<>]*?>\s*<strong>([^<>]*?)</strong>\s*</td>\s*<td>\s*Vehicle\s*Body\s*:\s*<[/]*?\s*BR[^<>]*?>\s*<strong>([^<>]*?)</strong>\s*</td>\s*<td>\s*Tag\s*Number\s*:\s*<[/]*?\s*BR[^<>]*?>\s*<strong>([^<>]*?)</strong>\s*</td>\s*</tr>)"))
                {
                    or.Vehicle += m.Groups[1].Value.Trim() + ";";
                    or.Vehicle += m.Groups[2].Value.Trim() + ";";
                    or.Vehicle += m.Groups[3].Value.Trim() + ";";
                    or.Vehicle += m.Groups[4].Value.Trim() + ";";
                    or.Vehicle += m.Groups[5].Value.Trim() + ";";
                    or.Vehicle += m.Groups[6].Value.Trim() + "~";
                }
                if (!string.IsNullOrEmpty(or.Vehicle))
                    or.Vehicle = "Make;Type;Color;Year;Body;Number~" + or.Vehicle;

                foreach (Match m in RegExpr.ParseMatchCollection(offenderHtml,
                   @"(?si:<tr[^<>]*?>\s*<td[^<>]*?>\s*Vessel\s*\s*Make\s*:\s*<[/]*?\s*BR[^<>]*?>\s*<strong>([^<>]*?)</strong>\s*</td>\s*<td[^<>]*?>\s*Vessel[^<>]*?Type\s*:\s*<[/]*?\s*BR[^<>]*?>\s*<strong>([^<>]*?)</strong>\s*</td>\s*<td[^<>]*?>\s*Vessel\s*Color\s*:\s*<[/]*?\s*BR[^<>]*?>\s*<strong>([^<>]*?)</strong>\s*</td>\s*</tr>\s*<tr[^<>]*?>\s*<td[^<>]*?>\s*Motor\s*Type\s*:\s*<[/]*?\s*BR[^<>]*?>\s*<strong>([^<>]*?)</strong>\s*</td>\s*<td[^<>]*?>\s*Hull\s*Material\s*:\s*<[/]*?\s*BR[^<>]*?>\s*<strong>([^<>]*?)</strong>\s*</td>\s*<td[^<>]*?>\s*Vessel\s*Year\s*:\s*<[/]*?\s*BR[^<>]*?>\s*<strong>([^<>]*?)</strong>\s*</td>\s*</tr>\s*<tr[^<>]*?>\s*<td[^<>]*?>\s*Registration\s*Number\s*:\s*<[/]*?\s*BR[^<>]*?>\s*<strong>([^<>]*?)</strong>\s*</td>\s*</tr>)"))
                {
                    or.vessel += m.Groups[1].Value.Trim() + ";";
                    or.vessel += m.Groups[2].Value.Trim() + ";";
                    or.vessel += m.Groups[3].Value.Trim() + ";";
                    or.vessel += m.Groups[4].Value.Trim() + ";";
                    or.vessel += m.Groups[5].Value.Trim() + ";";
                    or.vessel += m.Groups[6].Value.Trim() + ";";
                    or.vessel += m.Groups[7].Value.Trim() + "~";
                }
                if (!string.IsNullOrEmpty(or.vessel))
                    or.vessel = "Make;Type;Color;MotorType;HullMaterial;Year;Number~" + or.vessel;

                ResultFile.Write(or.GetValues());
            }
        }

        private void UpdateProgress()
        {
            if (Params.GetIntegerParam(DmConsts.StartPosition) > Params.GetIntegerParam(DmConsts.EndPositon))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.StartPosition), Params.GetIntegerParam(DmConsts.EndPositon),
                    DmConsts.StartPosition, DmConsts.EndPositon));
            if (Params.GetIntegerParam(DmConsts.LastPosition) > Params.GetIntegerParam(DmConsts.EndPositon))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.LastPosition), Params.GetIntegerParam(DmConsts.EndPositon),
                    DmConsts.StartPosition, DmConsts.EndPositon));
            if (Params.GetIntegerParam(DmConsts.StartPosition) > Params.GetIntegerParam(DmConsts.LastPosition))
                throw new Exception(String.Format("Invalid parameters {2}[{0}]; {3} [{1}]",
                    Params.GetIntegerParam(DmConsts.StartPosition), Params.GetIntegerParam(DmConsts.LastPosition),
                    DmConsts.StartPosition, DmConsts.LastPosition));
            UpdateProgress(((double)(Params.GetIntegerParam(DmConsts.LastPosition) - Params.GetIntegerParam(DmConsts.StartPosition))) / (Params.GetIntegerParam(DmConsts.EndPositon) - Params.GetIntegerParam(DmConsts.StartPosition) + 1));
        }
    }
}
